import random

import requests
import json
import pymysql
import time
from lxml import etree
#商品信息
from bs4 import BeautifulSoup
from selenium.common.exceptions import NoSuchElementException
import decimal

from com.py.test.reptleDolinGoodMsg.DbOperate import DbOperate
from com.py.test.reptleDolinGoodMsg.MyThread import MyThread
#
from requests.exceptions import RequestException

from com.py.test.reptleDolinGoodMsg.RequestReptle.TMReptle.TM_GoodsDetailAndComments import initReptile

dbo = DbOperate()
class TMGoodsUrlsReptle:
    def runThread(self,urls):
        for url in urls:
            initReptile(url)
    def start(self,baseUrl):
        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36"}
        goodsList = []
        #商品列表
        print(baseUrl)
        res = requests.get(baseUrl,headers=headers)
        res.encoding = 'utf-8'
        html = BeautifulSoup(res.text,"lxml")
        goods_list = html.find('div',id="J_goodsList")
        try:
            goods_list = goods_list.find_all('div',class_="gl-i-wrap")
        except AttributeError as e:
            print("没有商品列表了！")
            exit(0)
        #获取商品详情页地址
        for item in goods_list:
            goods_item = {}
            goods_item['goodsUrl'] = item.find('div',class_="p-img").a['href'].replace('https://','//').replace('//','https://')
            goods_item['goodsPrice'] = item.find('div',class_="p-price").find('strong').get('data-price')
            if not goods_item['goodsPrice'] :
                goods_item['goodsPrice'] = item.find('div',class_="p-price").find('i').text
            goods_item['goodsName'] = item.find('div',class_="p-name").find('em').text
            goodsList.append(goods_item);
        # print(goodsList)
        size = len(goodsList)
        step = 5 #5#一组
        groups = [goodsList[i:i+step] for i in range(0,size,step)]
        thread_num = len(groups)  #线程数
        for i in range(thread_num):
            t = MyThread(self.runThread,args=(groups[i]))
            t.start()

baseUrl = ""
tempNum = 0
for num in range(25):
    tempNum = tempNum+ 2
    TMGoodsUrlsReptle().start(baseUrl+str(tempNum))
##测试
# JDReptle().start()
